package com.crayon2f.test;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Before;
import org.junit.Test;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * Created by feifan.gou@gmail.com on 2020/10/28 11:52.
 */
public class CrawlingDZDP {

    @Getter
    @Setter
    public static class Info {

        private String id;
        private String name;
        private String address;
        private String phone;
        private String createTime;
        private String intro;
        private String openTime;
        private String specialServer;

        @Override
        public String toString() {
            return String.format("%s|%s|%s|%s", id, name, address, phone);
        }

        public Info(String id) {
            this.id = id;
        }

    }

    //城市ID
    private static final int CITY_ID = 8;
    //机构列表URL
    private static final String SHOP_LIST_URL = String.format("http://www.dianping.com/search/keyword/%d/0_培训机构/", CITY_ID);
    //机构详情URL
    private static final String SHOP_DETAIL_URL = "http://www.dianping.com/shop/";
    //爬取总页数
    private static final int TOTAL_PAGE = 50;
    //cookie信息
    private final static String COOKIE_INFO = "_lxsdk_cuid=175730bb65bc8-008c3c6f0cfa28-c781f38-1fa400-175730bb65bc8; _lxsdk=175730bb65bc8-008c3c6f0cfa28-c781f38-1fa400-175730bb65bc8; _hc.v=93af9bc2-1842-f08e-61eb-3b4845528786.1603955583; s_ViewType=10; ctu=793c64c9cc41f91e616e5c0ba3ca5b272418d856bd967db7966f8d15fe1aa323; Appshare2021_ab=shop:A:1; Hm_lvt_602b80cf8079ae6591966cc70a3940e7=1615184227; fspop=test; cy=8; cye=chengdu; _lx_utm=utm_source=shop_share; cityid=8; switchcityflashtoast=1; source=m_browser_test_22; m_flash2=1; default_ab=citylist:A:1|index:A:3|shopList:C:5; ll=7fd06e815b796be3df069dec7836c3df; ua=15010780830; citypinyin=chengdu; cityname=5oiQ6YO9; PHOENIX_ID=0a515a41-1781119ea52-80ef; _tr.u=VZVjmzz6DR2MAJLG; pvhistory=6L+U5ZuePjo8L3N0YXRpY3Rlc3QvbG9nZXZlbnQ/bmFtZT1XaGVyZUFtSUZhaWwmaW5mbz1odG1sLSU1QiU3QiUyMmNvZGUlMjIlM0ExJTJDJTIybWVzc2FnZSUyMiUzQSUyMlVzZXIlMjBkZW5pZWQlMjBHZW9sb2NhdGlvbiUyMiU3RCU1RCZjYWxsYmFjaz1XaGVyZUFtSTExNjE1MTk0NjE2MDYxPjo8MTYxNTE5NDYxNjA2MF1fWw==; Hm_lpvt_602b80cf8079ae6591966cc70a3940e7=1615340992; _lxsdk_s=17819d369f4-77f-fa0-202||14";
    //请求Template
    private static final RestTemplate TEMPLATE = new RestTemplate();
    //请求体
    private HttpEntity<String> entity = null;
    //临时路径
    private static final String TEMP_DIR = "C:\\Users\\goufeifan\\Desktop\\temp\\";
    //到处excel路径
    private static final String OUTPUT_EXCEL = TEMP_DIR + "DZDP.xlsx";
    //到处excel路径
    private static final String EXISTS_PATH = TEMP_DIR + "exists.txt";


    private Map<String, Info> getShopInfo() {

        Map<String, Info> map = new HashMap<>();
        for (int i = 0; i < TOTAL_PAGE; i++) {
            String url = SHOP_LIST_URL + "p" + (i + 1);
            ResponseEntity<String> exchange = TEMPLATE.exchange(url, HttpMethod.GET, entity, String.class);
            Document document = Jsoup.parse(exchange.getBody());
            Elements aList = document.getElementsByTag("a");
            aList.forEach(a -> {
                String attr = a.attr("data-shopid");
                if (Objects.nonNull(attr) && !attr.isEmpty()) {
                    map.put(attr, new Info(attr));
                }
            });
            System.out.printf("第%d页完成%n", i + 1);
        }
        System.out.println(map.keySet());
        return map;
    }

    @Test
    public void getShop() {

        Map<String, Info> shopInfo = getShopInfo();
        System.out.println(shopInfo);
    }

    @Test
    @SneakyThrows
    public void crawling() {

        List<String> exists = exists();
        Map<String, Info> shopInfo;
        if (exists.isEmpty()) {
            shopInfo = getShopInfo();
        } else {
            //请求过,需要记录下shipId
            Set<String> shopIds = Sets.newHashSet(
                    "k7ObwmGq8La02tul","H9Jg4P3nwpM3ptB2","l7FONhADfKbk47FF","k9FI9P1euSCotCl0","H5cqnOvOpGRL92bh","k6P5BjdjdR430FEe","G5vVa4Yfy1a6lT7B","k9w7Y0U8hUEGcT7M","H2TgNiXUa0er1EMs","l1qvKlCtjPi82cO8","l457Yo2lf4Apdn0I","l5sA4dpJAtJXh1x4","jywo7BVX8odPxQou","H7o0Q1GdAQTqQVo8","G7dLjK78rpDldCNJ","G5CTLglI5Ng40AaU","H5fn9kQGtObpFDg3","l1FT3wlRBfDCgLQK","Eq3m1pdQ8XSDvrmO","H9eU9YXbwzltfRDp","H6NijZieHG6lbLo4","k9GFddsjf67F4u7p","H7B2RULxxFz4qDpA","l8TbQg3OT2aKQnZU","i6rZY2LG83QkQ6Sc","l3nWWuePYrtb6GhW","EoykbrDiJY4tIDCU","l5nRpF3a3fF8eos6","G50eFnpnYBboHjIn","k7KmnBHPRZyvaapP","kaAMrPzvHc66JrOa","H9izvh5WoMGGUPR0","HafZfCvilOsQ3vRf","k7x1bWNyUgbpMsUj","la39JziaclhPs9qt","G5pmsUN4ktoDLjA1","G9L61sdAeUdRBnr9","G69zfXgD2zokpzdA","lavv17SCBtGH53ex","G6cxZS2aHmGLFvAk","H9qn6hW6rCotTapo","H2KYpHBQzWjUwGC4","H6RyRbk6kzqZNOVV","G9upzRyDGVYjkM4G","G3nEnRPCIJzO9oUj","l2GBorlivihbjvt9","k2JeoMsOoT3jeKBJ","E8JLGeImY0ymKjnc","k7VXRNL6yZj6NW9i","H5sEOaAjN8PffodG","l5tExzeOwL9cKwB6","k8u1092YIGcX3Z52","H9B2dH6qIv7nwbz7","G9Gh4diy0U6lhFXz","G4AHeefHGQ2HijXe","G8hh8szZxifXvUKu","laFb4SjJWwxgCu9z","k1thJIBxzqicFFUY","Ey3EDg23HQhyIvGj","G5PBJWOSTC5zXoht","G6bW8H1XscFT64BI","k6men1xIQWNWDAcX","Ha4PpFohzWVdS2ve","GaWmJYOmeaTKtVwu","k2JCADVGcLGKQGRF","l1fzr1q0BZRaMswP","H1fliVuUgobuCtKC","H3o736tie3j72TyQ","l58qsWLtozUap3f5","ihaW9UHdHLUm2GLf","ECrYeZICWMwV2VCK","k2lv0yBz8QH95l4V","Frqr9y11d2bKnSnk","G6Zqyefj85tVGOGE","G5X9uEpbNGswmP0l","k8LiqC89Zva7K52u","G4D4x4d1jGK0shoC","l2B8oRY3ONOW1eW1","l1gTFYFDQhRs9UPH","l2vl9arjb7W4EkQ0","l1ji1PlRoaVGLXlh","i8NyrS8eC4kS7f5I","G7R9uJB0Ig1rbyYz","k4nOMDiNPbsltLee","G5iYZ2lP0e8Ut6cT","l4pCbWsF52lBgVXc","H8KB0QtK0Iq83Fnt","l4F4SvSs6oUGqaFg","kaTYzxWSm4XVo27h","G3vkQwK2RQo2mBxW","ij0b1xDXFnPSTtNy","l9jkAvP2zVCvIxEt","k4UO05lrReACxNSA","k28HG3GNoFHBA5fN","j7iITEckWnEwk1Wv","H62ZG0DifXD1yFwX","jcJN8MBfg85DfwIZ","G4iPzgTMsoveJZ8G","l98SgLXIKY4nknbj","k9BcKLoaY4zVYy52","k2fVn4DF6cASYbL8","H7pG92tRU65hLU2x","k4wb9YhO5l9BdNrH","l7tPmEzTENGSZpCk","G586QeL6lIrvC3Pm","H9sM0fy108uBlp8G","l9BRFuWDgyHcKOEu","G2gmJ2it1lMRzIgK","l1NMSclrpebDMpg0","kaOScM1g6wc45n5q","k65pmL4Cgzk4j849","la8SEQjKtwcsOpCk","l3QCsLf7qTwmNJ1p","G9zNszlAzh8P0lnr","l2268pRgiW0dJ1xb","H4xZELYVTBdAmarH","H77ixHYuJYhMU3No","k8dWdk1M5l95rt32","G1H24K96o8mSoRk5","G6z7eb87zlaQMwNs","k6R16nouURLAqaum","H5HMa7zmqBz048EE","l95pSKAiaX04gaNR","H3Kcd5vWkqzQIgHX","l8Qly1YWIEhXiuCV","l4ULYIh6eJkvQmAY","GaPglub1525uL5NB","l5Qd4uhQyjpc3SJQ","FuB4tVZNVenxDYeX","l1toKykjyUJ8Qi1C","H15a5lfUbsjzWScj","HaAhmilAAOF0DtiK","l2DYSD4neFzQomm6","H5x7bvWhDco3hShw","laT5jC7kzrndeekc","G21oV97KjYJH5iLz","HaFbE36Y9IVkeeTC","G6YCfYWCDWtETKvN","H22MY02JwOT3nJLJ","H5RMyOxt5vcYV9Ul","G8FtQtmP749EG3vP","H2Jn0USkJc0OdEyW","k7kHRlrYTFfSQvaZ","G4hEj7QNFRblHsiO","l6KjkTmpEtQgrAaf","H6EDseaggorhN13d","k7iJXvzV42a3B6b6","H54DVWi6LzuXDrpa","laEsOcquHRhcoJxC","l2atAU4wC0ESxkRZ","ka1bSbns8tRFwGJs","H4Tu7IfYY6PTZdCu","k77INMKw1czYrKWW","H7946SPIaPNlgtDe","k6dNfADiOlmutcRE","iPQT9mtr1WqENeSa","H3nwgo42ikYtw1qK","iyBIQo5RoY7PFYYt","ijARcDCFrystA3jI","l6bUhd4lTLVL7JZq","GaCptKbC0M6s6Q3J","k45ZkS8i4XqTdFtx","l9I3xH2SvHHxULK7","l7IOKhfPy1mpRxpc","EwNo57KDCoiKjEDQ","l2iCFvhpk5eDJ3a8","G5f6suGNLJhlw3PO","k6JppkcbKDpIXTT1","H21WPtrMb5jjsDB0","G9MQEWDQIRfvt2rG","H9WZMHskypxaTksc","k5RmSpe9HjD5EJNN","H6fM06Jz2c7M0q2U","HaPWkFpDsk2bU7CN","G9oM7IbgcVntXI9q","k5tXzyrmF2IHeRUK","G7WgnIQ4EzeD0IkH","G3DLj0jJ4U91vGKo","Ha4jw98lANOeo4Zw","k6rp9Zq7JNlkjH2H","l8ckrGx5HZ7VSTXP","k843A4hhHSMB1pgj","k7QlTddNNMQoe2LU","l6go6PgI8nJSbl0r","l6JFxBYV8djibeY7","G3h60Xr0Q1kbTxl4","G2BUmGVqybmp5uIA","k2ZkVA6Q5h46qXY0","H6dbAowAOyGqDFFn","EShk3llJIHHJzhcM","G13CsJnSyeQaVHM0","k7UZvgC0sAx3FXGj","l4bBYS3AN9kPOvHN","k5XQpNopCyVNjvuq","H93s11pjTINFYXOG","l8AIF15Jht2HUXzx","H8MlrT9VAPCHRz18","l5SZTM9cyCeFVwKm","l4hTfIOpA2QruPpR","l5719IoE0757zwLc","H905s71Mlo6LHTNf","G4NRGqmnEPROjthu","FGNDQuGd35Db1Skg","k2KyvHE3DLffhxvQ","l8WgDaMjN71j7lY4","k1drXe9q4zoxf2uY","l9X0nF9zcevRbuII","k2KJF0Vkl7EGQZeB","H3R8W4BOweZa2XA1","G4JKzb08u6PUNBL1","k5QomG3GINrzMr9G","G5J4hQBg2KYG2oIO","H9JuAqEISyKoIGHB","l3gHv7h6Je8B1j92","l8Giy9FodEL0we7n","l5w1AOo7kFgyzSYe","l3362OttcZa3rQkk","l2AtKT1wIfGdIuOF","l6vHoF5F4ezQVcai","G6f8ojVQzy83voic","H5NBjySKyxCQ702O","l6neU22pUvZ1OrFv","G3fClUniuqaDbM3T","G6dYmtXXaJqyPcW5","H9uAHlBBUyUxPF86","G6vhlMv5f7fhP0bd","jat5bflrR99Vx8xT","H5hmYkCw0LUP8XHr","G7W7oqsRIlhIro8N","EUnrbH5yx6l4N9cR","l39kmvfS8RZP8Uzy","H6ZWkTHBISIayh1w","H3RqF4Z1R5BtDeBQ","H53Xdndba6ol5bcd","k708SHKJM7mAjF78","l8EuTPQMzCV8He1y","FS7dEGhMqxOix1rT","H9qvCNZGg3vlknPP","l7FljIiOCLqaKnnc","H5Tsom3Sk5wptYP1","k66ghIhncgzcSiCG","kayBe0Gr6AI7UTzP","H2wYYcrHkYPFrxlI","EbEjvphcOCbyq0UE","lafIEeLYUKuzBgGL","G90OXN8FGDfaSLVl","k7ydYkpZo89UsTOh","H1Benp7KmWBNzn7n","GaDtoa7VohXCxpJc","H5DDbp8umzVu1yQw","k834us5XQM7fNUsv","lajE0EbSP1psoh2H","H5O6ZgcbIV5u26Ka","i5BAR2xdzrPtW925","k9n9oqc3dU2A3C9W","ka66WQOhH4poPGTH","lahNGluu2tYQmD01","H5xu5HB5Y7WHCXaq","k10hWeyrt95JigV7","G42f1p4DhiQPIyDq","G8XXhVEIVdiXxHRQ","k5s0SZSnvURNyNFz","k46K5Jwvds7npFUy","l1EKDZq6b3OTiWLe","H81pzUObMwTR3Xxr","G19eNZp9t2tSibbw","H9If8PfBMHb7UXax","G6S78tyMQfK3mLxM","G5il2DJQxJf0JLzP","DHKMvzIZIDLCeSvt","H4MViOfakCbUUxBA","l9FmJRr0gnQGw235","k8nQUxHgoxKIG8T5","l6LInLyXAxxzIYV9","l7p5RoPUffJGRfSj","k78ElZibISoJUf71","H6fKCG5c4rcsQQHo","Ham0hZ4rLIoozjdY","H3Xq3U78L3xhQfun","l8keaG3YMptYPznc","k8RURM4vJChAOIIW","G9ynRvP2sxTwkVK9","l8TtA6nmodaMpl0I","H5bYgHzAM4De59RW","k9OBSvBuyLmtvWXE","iZNRu1djMqWEy7p7","Gan0sTcMhQ5TAncx","l5VVmAXWTnRuhetl","k3PT0hxBuqQBEGJU","jFtaLC9VB8vwHdxR","H5tNyvlG8V4DHa3v","G1D2cYEOaYWHl87q","iEtAwIpU8ZoCacHA","H2IQjuqCvpsyeIm6","G8qHTIhVV1FosMgf","l1YZsVvFtWJwsblR","GapGm0Qstfqe0yY3","k491BHmFz0Od2kYA","H7AlgDZKkPI81CO8","k5vucNAo1QPysHY4","k7U7ybNors7ra1Om","karDQzzBEHYAP7hu","H4T7c26pOn5iQ002","H9U18Cb1wZtjHl6l","H777lbJl3AAa9Wd8","H6uJfyYvypu5M0Th","G7WgkrvLjvgVQfRr","H95BamFOI6slMubo","G8RGerQHWoaSgv9V","k11g7POJb80B4wYI","GaADu9pOFQcwRPFl","H8vbSWSd7pKeH7oD","G706qkrXZjNQ8Eif","ETH2Ogg9pnLQaEqS","G1AE53gigHvgFuhD","l6WBX0nnTSNvDF8g","iD4EGLwUSbJ9o23K","l3wbv3TtzKKKxmSg","H4KZxZtk1Q2nIj7n","jnVATDciiQbscrha","lajKUMTAJTDJYxA5","H6iMPsR3SvUfKwzz","l2ZGc8GhJt5ps4lC","Ga8PBoxsTYGiqIXX","H3TLeYCKZIBkQcJZ","G9UW5gI9luVXxwR8","EJVPws8BVcWpvRww","H6dsQ7DK0B16Zs9t","k98HdedFTlvbK8nY","k6Q2eq8IwnbZmuRN","l1Y5pNyIYA2pDvHB","E2Qh5rOIT9lslA2K","l5xTCWSaWWcrnthX","l7Nq1i99l1DdYGm6","k92X094WAuRsCNvt","l68BQuZCza61ou54","l7m1s9Ke5cZoX6UM","GaVE6D6m2pVf3jUK","G20XHYXvJINrN7kO","G2hieEVlMHp4Ws6C","k9MSjKhXG8u7RjGR","l7alTtSNxIi28p14","FeGdieVchS84HkZP","l4O8Ak8LYfFZssE8","H3sxDDwf8avbYJpm","l2zhlFYSXuZlBdpx","H9YBr7DVAaxq9ndT","G6SxFYmGLPZ4xMJL","k2J6bBRrurnQoaZN","l4SX5RU2NTKliMy1","H9vFpaEgzEyA3shZ","G4IXBk8FuvrusAV3","G7x8D9YQFpLsR7Aw","l1MvtRDjx3NqyknQ","k4w1b7HylLzZeiN4","l7JcKqrCeCVMV5yc","G44NgaOxn8O6J3Jw","l1iYRwn70X6Uu2Vf","G5vPDXWCpPVQH9NN","k9xHbg00sgz6TL60","k2H67k9uG47p4YfD","l3ZvcBqdwbyENkMj","H3nf3McPWiX1tymr","G6seJRyyR7ptOBEt","l7i4tkvXB5BUvY8W","H73kPs8uQURE36pc","FGLlkY1efdQKunPc","H5zxtyzrZPuRrNEi","iZtmZSgXxwiihiaP","l9Xk4NNp7BRJHrvK","FOyrd3Mq8jURE1X5","G6jHhyGZIU4QyJwu","l3mrZBry2Qugubil","k3H54GnYNf8hTR4V","k4qCdqQBkzreu1tz","EoNsAaSQMw4S9rL4","H4IbKhVScqw5rkTv","H82kZ1mMeGDco4eb","G5CJUSeORescVGFv","GaOmhfM0ei2HRo1I","Eea0tNJm0rYEp5aa","jmUzNRinfznqN8Up","ih7LaFfakdj5QJ7x","l7uVNSabWbwMo8Kt","G3H0NMTa8UrF1dti","G2Yg96yYTPGC7iZM","k7qYF139Nf4F2O9V","G9lnrJZEX3WnHoDO","l8cdBaDxTKAAptJG","H1AWmoOzd1CRap7T","k4jEAYR6JPu8MGtv","H43dQ6uKwICBTTZP","G8GTXeGl52r6GOqw","k4pZo7eVyIsqMfLs","l6Y1WhfyTxjZFihL","Ga8LtOBLqC35R7uG","k47QicPLkqIL12y7","HaPjOLtq9UsWvrU9","G4tkbZMhCheat7uv","H6B5N8g9Uc6ZOedi","l8IMQpGaolOBrXVO","jwQEjcXFQU6nDTQG","k8x5lLRJU5Gfledy","H2OpaPugyOXZuMGw","GaJs1Fl7vQvnrDAj","El7hfcX3jWWZh8Jd","ibvrGWVvznMwYFwU","i2DcfYzvUHs6M48m","H4PGNjeTRZqV4zp6","l8Ucaw1chxtQDYRC","k9nkpV3CeGSyCtyH","G5lYFI2ailKRf9Zu","H18lXtMymhyTxHqq","H6BOmacaoblwdv7w","H4F41XlR4zNb0msB","jndvd7CWAywju9eO","l7qipUbrHdCn6Zof","H4dZZfJ5kctXHt9P","l5O0dl5Bo7B9VE4l","G266x6rvXRIpQPgO","k121DYINNuFCTHyl","H64hETxb6HCAt0sd","l3JBQbwnTndVTKPw","k1Y64N1xDQJtI8DC","G34boZTx3bYR9spS","GaWJfP8eW8ByrWAT","G7xECv5arQAFpk1U","l8Fl64v88MDoi0qk","k67dkhDHGOJY6CHe","G97rQ10HnXwZa6fY","l5bAVg6cC0nr8bUN","H7v2UQURJRtyKCyA","l6DHP3OUzgBordFH","l3tMEL4SvXedgcrw","l7KOEE2793ZjegsB","l9abMqAYxYjj68Aq","G8NLMKDWIvWJUSL9","G4hPNVZB2bXUKvwS","H5x3unsm14KldSAN","G3hArD0SSKaRzVNp","H6K5THW0cK2gufFq","H6IImTXb2ydj5Fyv","G9RNn4R9qBeEubKm","l78orxZIQbcW1t7G","G1IcsGKvyoqCbIic","k5oeYQkvDYstQouE","k3I6enjorldeNhWb","H9LDAOHQYw8oMvii","k1EFiQCHTENwspu6","k3GugXVR8WdsbjgX","F8UBMt2iMLWSgpkZ","k7GbddOqEjQFUt44","G5A6ZbU81mKUL1T9","H4ViJ9Z1rx17wigz","l5nt0qAoyYqNV5aX","G7D2aE96LGC4GqJM","Ft04ai7jNJuALVp6","k1pVRCdRPnQzGNNj","jYBxWDNO1Y9DxVfL","l2qp33ep76tNAPjs","G4e6HS1a4LguvcnQ","l4R5oPkfbU4PhR2r","l8hQgppAK868sgsR","FfvZCHU8ZWJlHb5a","EUuHGjENScZTKq2N","l2cTYmQPbYDBRMZE","G1mMuAF4WeLQGSE5","k1cNuUrhBXtVmWa4","H5x3VCS0whsM0z9i","l6NccVUfW3QeVJvJ","l2AVRUOAeTv1EXIV","l1OB145JgpBYxsaf","iI9Yysm8s08zKH7q","H7tSBG9YaJRirIYE","l3orXQvWUbThgxtg","G9BiiAlciSXpo8dl","l5cx93j8R18zM30D","l6TAHSqDP9t0mQiH","G4JrurHNHVROFTO6","G3RVeXD8K6i6sCku","G91we9CWymMgzXW0","H1eIBzHgauGodm1E","G6A8GjLWZLRHdRDu","iR1gIebQKnzgiXg2","H3QRuwf87u04Bc38","iKAepB96wi2YMmyc","H7fWDjegoHVgw59V","l7GPki77FEmkVHWe","l8gAUVG3VD3CnIz9","G3GBJ6RyWJOmmswc","G5MZELpAQjQkmhbP","GaKB5AlJbBo4dTMd","H5xPQ9rHvHXZBonN","G1mMzDEQOejxeWUC","G8abm5avHFgmqiRN","kaOG25tqQFBlZ370","k1zJ0l48fWc4ZRjD","H9s0CvmdhtTrhlGR","k2oKyxjy30zu0ptN","l85vi8d6qnAWa7rU","H9QFCAq7dSqwu6ds","H4X042gD8HwuWxCv","jKDgMfaZYWXnWjOk","H7lgGPpYBL0yWRDH","H3NJQgEKPS1ii5FE","k4Epr9tyKAk98Yv0","l2MpjM2h12UE8K4I","G1kKZKE65L4eYGVR","G4wEzCwVn6XsB6gG","HamBWp4ZRrcHo2zc","G63KRGb3SZiVIIAy","G97CnAnWH8eUtcbu","jBihxRNUXrhQYIeo","G5vlTe1qt1HBH6pZ","H5ES8ciqb6GsdqoY","l1XCIqjvg5lZ63oM","H1Axz6jQPgNI0aRp","ix2q38x2i6ju03IW","laDfPoliyzhIsUmK","G3bkU7moPeA2icFk","k8Zej2yfcVIEwLO9","G7Z00mplV6omziiA","l3pxvxEeGERTr8ln","l5s8XNn5viLtoMfv","G42zrCFLxRcahUzB","H2ab923dXnZTzxXp","l9Vgrrx4TYp63ui2","jYXCk3ty27T81Dn9","H9ndnulTUCAjfdTQ","k4MJJrl23jA2kenz","H8liaietjxxYw7Kf","G36fKf48WoyUklTa","GaTITx9JGfeqJ9AI","k4Oheglma74WZ6L0","H3v9rQjB5oecDaiC","ErKigDM0Fj6btItz","k9sjB3I94igX6q1j","H5bCIOsXAoiix1G6","l14hGDXvxqD8J1Ko","k9zbAZFjyL8udqux","l863RB8jgSWPCbuo","G3mgRVPFdlZLDG8G","k3PoeDhoTr1LRXy2","FJeq95JDzG88Qu0H","kaOzAk24euifKHua","H9xdJFrXKtX51Ero","l6xy4vJzXL9kMXJs","l1vlTnJU0IqGF6jQ","l9JzRxwpG3o89Yug","kapwBfPxPj5YYyQK","G1gBvfpZEQUUM0bx","G5hVosrpYbwiycSE","k4WWbq6buCpRpEke","G7EHSsD23FfIftE8","G8J2r6QtdX1c6WUh","k5pkmOoM7YHTzjoA","G1r7n3cieLv2MS16","l7wDmBF7lPtLepwm","k5pUbVDFBAH0qTUW","H6skUittaDXUbWUf","iFdkDFzd1Rx3wVnz","Haw02mRUxS0aMX2a","k3iQJhD5zya1x8j0","k7XMW6vJ5FzoIKAY","GaKReEMRF0muMncY","k6tvOJL1MLQeVWAe","l2D1Y9B9sw0usR5R","H6S8tNGwyUZN1e09","H3PuQMCULe6oWhJv","l68OP8UL02sFfSZX","laK6d8lLaPcMWlQP","H9nQLbzulCnsls0x","k2QzbuxK1NkxKyYm","H3Ld2C7qXaxtd3jh","k9b4LTr5rpz2Lqcv","G9mGtL0c7OyqwO4W","k1MD3NljURoQB7iT","HaegZjHBohCpjHlF","ipxu2hhyjp9DXwYV","EKJS3HJoWVRcj1VC","H3N3vle0rOtUrl06","k3KGTwf45dk1eKL0","l20kCHFVC1aRb8un","H1vPOIObJesuqKk6","FbiOUnbhTz4tJRgR","G7UQ13UBxclfRkkm","k8jctOpJkqgEsEYu","G4hUsppcdkzL5OS6","G7PLVmeteLTowFKv","l44HtFbdJFpgNvvj","l2AIpE4kDm39u0UY","l21d5CaC3jHf4Smz","l5sDud181VGO1Ztl","iRT3VB783vdpLdD1","l7yuJN4LR6icyfnw","l27akVuykC4rEaHn","l4JHLtgDyQQvkjuL","l8Z4Tlr328crrnzc","k3OR0XKmB50dQGYb","G9uyt7a6251NcPum","laSwzx7bR9KJNmzS","lajBaiBDAmozPgEt","l5k00xSq9ckbnzTj","G16vgOgSycwydUms","G419OZyCRPx0E2pS","kabNDdveD0QLOqZg","k8uxyIk7ebiUw9qF","G63xsLMC9VHICb2f","G4rFaow7RwzXELnc","G4BvLDj7uySEyyJU","HaMeY7DKgOE0A6s3","l6KQL5PbSVDlo6Ey","H8lmfjL80RVEuXiC","l5wSUVV2fkZ13s4x","H1GjsgjxVCcS9T39","laEStdsLs0By6tC8","H9WoP6Mr2ihvUgcI","G2loRhqKz8pkxzJS"
            );
            shopInfo = new HashMap<>();
            shopIds.forEach(shopId -> shopInfo.put(shopId, new Info(shopId)));
        }
        int[] index = {0};
        XSSFWorkbook workbook = new XSSFWorkbook();
        XSSFSheet sheet = workbook.createSheet();
        shopInfo.forEach((shopId, info) -> {
            if (exists.contains(shopId)) {
                return;
            }
            try {
                ResponseEntity<String> exchange;
                int jj = 0;
                while (true) {
                    try {
                        if (jj > 100) {
                            write2Excel(workbook);
                            System.exit(0);
                        }
                        exchange = TEMPLATE.exchange(SHOP_DETAIL_URL + shopId, HttpMethod.GET, entity, String.class);
                        jj = 0;
                        break;
                    } catch (Exception ignored) {
                        jj++;
                    }
                }
                Document document = Jsoup.parse(exchange.getBody());
                Elements phone = document.getElementsByClass("phone");
                if (CollectionUtils.isNotEmpty(phone)) {
                    Element element = phone.get(0);
                    Elements spans = element.getElementsByTag("span");
                    if (CollectionUtils.isNotEmpty(spans)) {
                        String phoneNumber = spans.stream()
                                .map(ths -> ths.attr("data-phone"))
                                .filter(StringUtils::isNoneBlank)
                                .collect(Collectors.joining("，"));
                        info.setPhone(phoneNumber);
                    }
                }
                Elements address = document.getElementsByClass("address");
                if (CollectionUtils.isNotEmpty(address)) {
                    String addressStr = address.stream()
                            .filter(ths -> ths.is("div.address"))
                            .findFirst()
                            .map(ths -> StringUtils.trimToEmpty(ths.text()))
                            .map(ths -> ths.substring(ths.indexOf("：") + 1))
                            .orElse("暂无");
                    info.setAddress(addressStr);
                }
                Elements shopName = document.getElementsByClass("shop-name");
                if (CollectionUtils.isNotEmpty(shopName)) {
                    info.setName(shopName.get(0).getElementsByTag("h1").get(0).text());
                }
                Elements con = document.getElementsByClass("shop-info");
                if (CollectionUtils.isNotEmpty(con)) {
                    Elements lis = con.get(0).getElementsByTag("li");
                    for (int i = 0; i < lis.size(); i++) {
                        Element li = lis.get(i);
                        String text = li.text().substring(li.text().indexOf(" ") + 1);
                        switch (i) {
                            case 0:
                                info.setCreateTime(text);
                                break;
                            case 1:
                                info.setIntro(text);
                                break;
                            case 2:
                                info.setOpenTime(text);
                                break;
                            case 3:
                                info.setSpecialServer(String.join("|", text.split("(\\s)+")));
                                break;
                        }
                    }
                }
            } catch (Exception e) {
                System.out.println(info.getName() + "============= error");
                e.printStackTrace();
            }
            if (StringUtils.isEmpty(info.getName())) {
                write2Excel(workbook);
                System.exit(0);
            }
            write2Sheet(info, index[0], sheet);
            System.out.println(info);
//            try {
//                TimeUnit.SECONDS.sleep(new Random().ints(1, 20, 30).max().orElse(15));
//            } catch (InterruptedException e) {
//                e.printStackTrace();
//            }
            index[0]++;
        });
        write2Excel(workbook);
    }

    private void write2Sheet(Info info, int index, XSSFSheet sheet) {

        if (StringUtils.isNotBlank(info.getName())) {
            XSSFRow row = sheet.createRow(index);
            XSSFCell id = row.createCell(0);
            id.setCellValue(info.getId());
            XSSFCell name = row.createCell(1);
            name.setCellValue(info.getName());
            XSSFCell address = row.createCell(2);
            address.setCellValue(info.getAddress());
            XSSFCell phone = row.createCell(3);
            phone.setCellValue(info.getPhone());
            XSSFCell createTime = row.createCell(4);
            createTime.setCellValue(info.getCreateTime());
            XSSFCell intro = row.createCell(5);
            intro.setCellValue(info.getIntro());
            XSSFCell openTime = row.createCell(6);
            openTime.setCellValue(info.getOpenTime());
            XSSFCell specialServer = row.createCell(7);
            specialServer.setCellValue(info.getSpecialServer());
        }
    }

    private void write2Excel(XSSFWorkbook workbook) {

        try {
            File excel = new File(OUTPUT_EXCEL);
            if (!excel.exists()) {
                excel.createNewFile();
            }
            workbook.write(new FileOutputStream(OUTPUT_EXCEL));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private HttpEntity<String> buildEntity() {

        HttpHeaders headers = new HttpHeaders();
        headers.put("Cookie", Pattern.compile("(; )").splitAsStream(COOKIE_INFO).collect(Collectors.toList()));
        headers.add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36");
        headers.add("Connection", "keep-alive");
        return new HttpEntity<>(null, headers);
    }

    @SneakyThrows
    public List<String> exists() {

        File existsTxt = new File(EXISTS_PATH);
        if (!existsTxt.exists()) {
            existsTxt.createNewFile();
        }
        return Files.readAllLines(Paths.get(EXISTS_PATH));
    }

    @Before
    public void before() {

        File dir = new File(TEMP_DIR);
        if (!dir.exists()) {
            dir.mkdirs();
        }
        entity = buildEntity();
    }

    @Test
    @SneakyThrows
    public void fix() {

        XSSFWorkbook workbook = new XSSFWorkbook();
        XSSFSheet sheet = workbook.createSheet();
        List<String> strings = Files.readAllLines(Paths.get(TEMP_DIR + "fix.txt"));
        int index = 0;
        for (String string : strings) {
            String[] split = string.split("(\\|)");
            Info info = new Info(split[0]);
            info.setName(split[1]);
            info.setAddress(split[2]);
            info.setPhone(split[3]);
            write2Sheet(info,index, sheet);
            index++;
        }
        write2Excel(workbook);
    }
}
